# -*- coding:UTF-8 -*-
from scrapy.spider import BaseSpider
from scrapy.http import Request
#from scrapy.selector import HtmlXPathSelector
from scrapy.selector import Selector
from scrapy.http import HtmlResponse
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.contrib.spiders import CrawlSpider, Rule
from lianjia.items import LianjiaItem
from scrapy.utils.response import get_base_url
from scrapy.utils.url import urljoin_rfc
import string,re
from scrapy.utils.response import open_in_browser

class LianjiaSpiderSpider(CrawlSpider):
    name = 'lianjia-spider'
    allowed_domains = ['lianjia.com']
    start_urls = ['http://bj.lianjia.com/ershoufang/pg1/']
    deeppath=0
    #rules = (
    #    Rule(SgmlLinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
    #)

    #def parse_item(self, response):
    #    hxs = HtmlXPathSelector(response)
    #    i = TianyaItem()
    #    #i['domain_id'] = hxs.select('//input[@id="sid"]/@value').extract()
    #    #i['name'] = hxs.select('//div[@id="name"]').extract()
    #    #i['description'] = hxs.select('//div[@id="description"]').extract()
    #    return i
        
    def parse(self, response):
        self.deeppath = self.deeppath + 1
        
        #hxs = HtmlXPathSelector(response)
        base_url = get_base_url(response)
        
        #sites = hxs.select(r'//ul[@id="house-lst"]/li/div[@class="info-panel"]/h2/a/@href').extract()
        #for url in sites:
        #    full_url = urljoin_rfc(base_url,url)
        #    #print full_url
        #    yield Request(full_url ,callback=self.parsenew)
        #nextpage = hxs.select(r'//div[@id="main"]/div/div[@class="links"]/a[@rel="nofollow"]/@href').extract()

        #for url in nextpage:
        #    full_url = urljoin_rfc(base_url,url)
        #    #print full_url
        #    yield Request(full_url ,callback=self.parse)
        house_list = response.selector.xpath(r'//ul[@id="house-lst"]/li')
        for house in house_list:
            #open_in_browser(house)
            #item = LianjiaItem()
            #item['name'] = house.xpath(r'.//div[@class="info-panel"]/h2/a/text()').extract()
            #item['region'] = house.xpath(r'.//a[@class="laisuzhou"]/span/text()').extract()
            #item['zone'] = house.xpath(r'.//span[@class="zone"]/span/text()').extract()
            #item['meters'] = house.xpath(r'.//span[@class="meters"]/text()').extract()
            #item['orientation'] = house.xpath(r'.//div[@class="where"]/span')[-1].xpath(r'.//text()').extract()
            #item['price'] = house.xpath(r'.//div[@class="price"]/span/text()').extract()
            #item['price_pre'] = house.xpath(r'.//div[@class="price-pre"]/text()').extract()
            #item['desc'] = house.xpath(r'.//div[@class="con"]/text()').extract()
            #item['link'] = house.xpath(r'.//div[@class="info-panel"]/h2/a/@href').extract()
            url = house.xpath(r'.//div[@class="info-panel"]/h2/a/@href').extract()
            if(len(url) <= 0):
                continue
            full_url = urljoin_rfc(base_url,url[0])
            yield Request(full_url ,callback=self.parsenew)
            #yield item
        num = string.atoi(response.url.split("/")[-2][2:])
        nextpage = 'http://bj.lianjia.com/ershoufang/pg'+str(num+1)+'/'
        #print num, nextpage
        print "****page ",num,"********deeppath=",self.deeppath,"***************"
        yield Request(nextpage, callback=self.parse)
        

    def parsenew(self, response):
        #hxs = HtmlXPathSelector(response)
        #base_url = get_base_url(response)
        hxs = response.selector
        
        profix = "coordinates: ["
        s = response.body.find(profix)
        if(s == -1):
            return
        s += len(profix)
        e = response.body.find("],",s)
        if(e == -1):
            return
        coords = response.body[s:e].split(", ")
        print coords

        infos = hxs.xpath(r'//div[@class="desc-text clear"]/dl')
        if( len(infos) != 8):
            return 
        
        item = LianjiaItem()
        tmp = hxs.xpath(r'//h1[@class="title-box left"]/text()').extract()
        item['title'] = "".join(tmp)
        tmp = infos[0].xpath(r'descendant::text()').extract()
        tmp = "".join(tmp).replace("售价：".decode("UTF-8"),"").split("/")
        item['price'] = re.search(r"\d*\.?\d+万".decode("UTF-8"),tmp[0]).group()
        item['meters'] = re.search(r"\d*\.?\d+㎡".decode("UTF-8"),tmp[1]).group()
        tmp = infos[1].xpath(r'descendant::text()').extract()
        item['price_pre'] = "".join(tmp).replace("单价：".decode("UTF-8"),"")
        tmp = infos[2].xpath(r'descendant::text()').extract()
        item['first_pay'] = "".join(tmp).replace("首付：".decode("UTF-8"),"")
        tmp = infos[3].xpath(r'descendant::text()').extract()
        item['month_pay'] = "".join(tmp).replace("月供：".decode("UTF-8"),"")
        tmp = infos[4].xpath(r'descendant::text()').extract()
        item['region'] = "".join(tmp).replace("户型：".decode("UTF-8"),"")
        tmp = infos[5].xpath(r'descendant::text()').extract()
        item['orientation'] = "".join(tmp).replace("朝向：".decode("UTF-8"),"")
        tmp = infos[6].xpath(r'descendant::text()').extract()
        item['floor'] = "".join(tmp).replace("楼层：".decode("UTF-8"),"")
        tmp = infos[7].xpath(r'descendant::text()').extract()
        item['name'] = "".join(tmp).replace("小区：".decode("UTF-8"),"")
        item['link'] = response.url
        item['coordx'] = coords[0]
        item['coordy'] = coords[1]
        yield item
    

        

        
